import org.apache.spark.ml.linalg.{Matrix, Vectors}
import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.ml.stat.{ChiSquareTest, Correlation}

/**
  * @author pinker on 08/02/2018
  */
object Correlation {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName("correlation")
      .master("local[*]")
      .getOrCreate()
    val df = spark.sparkContext.textFile("src/main/scala/Correlation.scala")
    println("-")
    df.foreach(println)
  }

  def correlation(spark: SparkSession): Unit = {
    val data = Seq(
      Vectors.sparse(4, Seq((0, 1.0), (3, -2.0))),
      Vectors.dense(4.0, 5.0, 0.0, 3.0),
      Vectors.dense(6.0, 7.0, 0.0, 8.0),
      Vectors.sparse(4, Seq((0, 9.0), (3, 1.0)))
    )
  }

  def test(): Unit = {
    val data = Seq(
      (0.0, Vectors.dense(0.5, 10.0)),
      (0.0, Vectors.dense(1.5, 20.0)),
      (1.0, Vectors.dense(1.5, 30.0)),
      (0.0, Vectors.dense(3.5, 30.0)),
      (0.0, Vectors.dense(3.5, 40.0)),
      (1.0, Vectors.dense(3.5, 40.0))
    )
  }
}
